import scrapy

import logging
import pandas as pd
from urllib.parse import quote
from copy import copy, deepcopy
import json
from HifoEsf.items_gp import GP01Item, GP02Item, GP03Item, GP04Item, GP05Item

logger = logging.getLogger(__name__)

district_li = [
    {'id': 'd2becaa5-7bb9-4c06-8d48-c36ac55b9707', 'name': '巴南区', 'simpleName': '巴南',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '巴南区,巴南'},
    {'id': '9e67eba2-5e89-40c9-a6b2-ed5d5606f8c3', 'name': '黔江区', 'simpleName': '黔江',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '黔江区,黔江'},
    {'id': '001e075b-860e-4e8c-9942-5d5e7f8b3a52', 'name': '九龙坡区', 'simpleName': '九龙坡',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '九龙坡区,九龙坡'},
    {'id': '92f49cf8-8ecc-4551-a3f0-db0967d71908', 'name': '涪陵区', 'simpleName': '涪陵',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '涪陵区,涪陵'},
    {'id': 'c065613d-196a-4e58-a23b-349b9145adad', 'name': '綦江区', 'simpleName': '綦江',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '綦江区,綦江,万盛区,万盛,万盛经济技术开发区'},
    {'id': 'eed378fe-5665-403a-8777-50c4ab5abc02', 'name': '渝北区', 'simpleName': '渝北',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '渝北区,渝北'},
    {'id': '13719bc8-7cd0-43e6-bb36-b9a8158721ee', 'name': '南岸区', 'simpleName': '南岸',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '南岸区,南岸'},
    {'id': '9ff0fe20-b625-4789-8be5-4afdf967f735', 'name': '长寿区', 'simpleName': '长寿',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '长寿区,长寿'},
    {'id': '7f651ffb-b410-4661-b150-e5fe95ba09ea', 'name': '江北区', 'simpleName': '江北',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '江北区,江北'},
    {'id': 'd655dccc-3491-4e89-b7ae-bca2b5d760da', 'name': '北碚区', 'simpleName': '北碚',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '北碚区,北碚'},
    {'id': 'd4cc4baa-95f2-4965-a9b3-dacf729b1260', 'name': '万州区', 'simpleName': '万州',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '万州区,万州'},
    {'id': 'ad3ae596-f3ae-4c78-937e-3421bc566beb', 'name': '渝中区', 'simpleName': '渝中',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '渝中区,渝中'},
    {'id': 'f2a89338-708b-47bd-a69d-1e191ab3b6f1', 'name': '沙坪坝区', 'simpleName': '沙坪坝',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '沙坪坝区,沙坪坝'},
    {'id': '4138c796-2663-4afa-bf58-e95bad12e848', 'name': '大渡口区', 'simpleName': '大渡口',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '大渡口区,大渡口'},
    {'id': 'e0fbd466-de20-4af1-a768-54077a80d004', 'name': '大足区', 'simpleName': '大足',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '大足区,大足,大足县,双桥区,双桥,双桥经济技术开发区'},
    {'id': '88d74bb7-b24b-4b59-85c7-1af4b623f4c8', 'name': '江津区', 'simpleName': '江津',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '江津区,江津'},
    {'id': '766c9def-af63-44b1-9a69-7165115526d2', 'name': '丰都县', 'simpleName': '丰都',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '丰都县,丰都'},
    {'id': '12abc274-28a8-48c2-8884-ee4ee8bf5442', 'name': '酉阳土家族苗族自治县', 'simpleName': '酉阳',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '酉阳土家族苗族自治县,酉阳,酉阳县'},
    {'id': '2ad17b65-66db-49bc-91ae-9386446dc41a', 'name': '彭水苗族土家族自治县', 'simpleName': '彭水',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '彭水苗族土家族自治县,彭水,彭水县'},
    {'id': '6b72787a-f357-40a4-ba32-49b3d4e6850d', 'name': '开州区', 'simpleName': '开州',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '开州区,开州,开县'},
    {'id': '38d415d5-f2fd-42aa-93a9-5e3e1cadec07', 'name': '石柱土家族自治县', 'simpleName': '石柱',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '石柱土家族自治县,石柱,石柱县'},
    {'id': '51c3fd22-1431-4b65-8424-5871e3792b39', 'name': '奉节县', 'simpleName': '奉节',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '奉节县,奉节'},
    {'id': 'dffd1ba2-a8d0-40fb-8a5a-adcf759c2700', 'name': '合川区', 'simpleName': '合川',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '合川区,合川'},
    {'id': '488cd447-06ea-4c36-b7ac-26734b09ffc8', 'name': '铜梁区', 'simpleName': '铜梁',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '铜梁区,铜梁'},
    {'id': '57e6eb97-5cb7-46a2-abd2-400d79f2b48a', 'name': '璧山区', 'simpleName': '璧山',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '璧山区,璧山'},
    {'id': 'e01b8034-405f-4235-ba74-7cf342b0df36', 'name': '巫溪县', 'simpleName': '巫溪',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '巫溪县,巫溪'},
    {'id': 'a91572bf-e143-4155-b1e8-0826b537f6d2', 'name': '永川区', 'simpleName': '永川',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '永川区,永川'},
    {'id': 'f05b86f2-139f-4307-b846-5960ee471945', 'name': '武隆区', 'simpleName': '武隆',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '武隆区,武隆县,武隆'},
    {'id': '562bfb92-9fc4-48c8-ba45-c895aa42972f', 'name': '忠县', 'simpleName': '忠县',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '忠县,忠县'},
    {'id': 'ff4d0830-0a50-4051-bc86-25494279ea56', 'name': '潼南区', 'simpleName': '潼南',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '潼南区,潼南'},
    {'id': '7dcb4143-2600-46ca-8d80-9f34ed6a6121', 'name': '云阳县', 'simpleName': '云阳',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '云阳县,云阳'},
    {'id': 'a9561c82-db98-4d20-95e2-e7f1b24b7801', 'name': '秀山土家族苗族自治县', 'simpleName': '秀山',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '秀山土家族苗族自治县,秀山,秀山县'},
    {'id': '2ae9c253-d6d5-4dac-b22e-66b0867f08b0', 'name': '城口县', 'simpleName': '城口',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '城口县,城口'},
    {'id': 'ececcd63-9f69-4fdd-af62-476d00eca6be', 'name': '梁平区', 'simpleName': '梁平',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '梁平区,梁平,梁平县'},
    {'id': 'fe81e222-e9bd-43fb-b9cb-b0d441243af1', 'name': '巫山县', 'simpleName': '巫山',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '巫山县,巫山'},
    {'id': 'c3b1d6ef-4d12-4b50-ae3a-d99571678ea2', 'name': '垫江县', 'simpleName': '垫江',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '垫江县,垫江'},
    {'id': '12f42b8f-6785-4ee3-a0eb-fa1593acd827', 'name': '南川区', 'simpleName': '南川',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '南川区,南川'},
    {'id': '05bbc290-5531-48a7-8676-e4f2c21ecd29', 'name': '荣昌区', 'simpleName': '荣昌',
     'parentId': '67d9a93b-b91b-441f-aec8-ba66356d9b8f', 'otherName': '荣昌区,荣昌,荣昌县'}
]

name_li = []
df = pd.read_excel(open(r"C:\Users\Administrator\Desktop\projects\HifoEsf\HifoEsf\csv\white.xlsx", 'rb'),
                   sheet_name='系统主城白名单楼盘')
df = df.loc[df['完成人'].isin(name_li), ['行政区', '楼盘名称', ]]
loupan_li = df.to_dict(orient='records')


class GpwSpider(scrapy.Spider):
    name = 'gpw'
    allowed_domains = ['183.230.7.247']
    start_urls = ['http://gpw/']
    # 查询url
    gp01_url = 'http://183.230.7.247:8899/estate/listEstate?page=1&limit=10&cityId=67d9a93b-b91b-441f-aec8-ba66356d9b8f&districtId={}&name={}&isDeleted=0'
    # 楼盘详情url
    gp02_url = 'http://183.230.7.247:8899/estate/detail?id={}'
    # 楼栋列表url
    gp03_url = 'http://183.230.7.247:8899/building/list?page=1&limit=90&communityId={}&isDeleted=0'
    # 住宅类别url
    gp04_url = 'http://183.230.7.247:8899/residenceType/listResidence?page=1&limit=10&communityId={}'
    # 面积段均价url
    gp05_url = 'http://183.230.7.247:8899/avgPrice/list?page=1&limit=90&communityId={}'
    # 待建楼盘url
    gp06_url = 'http://183.230.7.247:8899/built/listBuilt?page=1&limit=10&cityId=67d9a93b-b91b-441f-aec8-ba66356d9b8f&districtId=&name={}'
    custom_settings = {
        'DOWNLOADER_MIDDLEWARES': {
            'scrapy.downloadermiddlewares.retry.RetryMiddleware': None,
            "HifoEsf.middlewares.CustomRetryMiddleware": 500,

            'HifoEsf.middlewares.UserAgentMiddleware': 544,
            # 'HifoEsf.middlewares.OuterNetProxyMiddleware': 545,  # 启用redis中的代理，需要同时开启 RedisConnPipeline
        },
        'DOWNLOAD_DELAY': 0.1,
        'CONCURRENT_REQUESTS': 10,
    }

    def start_requests(self):
        """
        构造请求，获取楼盘基础信息
        :return:
        """
        for loupan_dict in loupan_li:
            district_name = loupan_dict['行政区']
            district_id = None
            for district_dict in district_li:
                if district_dict['name'] == district_name:
                    district_id = district_dict['id']
                    break
            loupan_name = loupan_dict['楼盘名称']
            if district_id:
                yield scrapy.Request(
                    self.gp01_url.format(district_id, quote(loupan_name)),
                    meta=dict(loupan_name=deepcopy(loupan_name), district_name=deepcopy(district_name), ),
                    dont_filter=True,
                )
            else:
                logger.error(f'{loupan_name} 行政区id提取失败')

    def parse(self, response, **kwargs):
        """
        获取楼盘基础信息
        :param response:
        :param kwargs:
        :return:
        """
        district_name = copy(response.meta['district_name'])
        loupan_name = copy(response.meta['loupan_name'])
        try:
            resp = json.loads(response.body.decode())
            data = resp['data']
            assert len(data)
            data_dict = dict()
            if len(data) > 1:
                for data_iter in data:
                    if data_iter['districtName'] == district_name and data_iter['name'] == loupan_name:
                        data_dict = data_iter
                    else:
                        continue
            else:
                data_dict: dict = data[0]
            assert data_dict
        except:
            item01 = GP01Item()
            item01['districtName'] = district_name
            item01['name'] = loupan_name
            item01['isFind'] = False
            item01['isBuilt'] = False
            yield item01
            logger.error(f'{district_name}-{loupan_name} 数据库没有该楼盘，或者楼盘不唯一')
        else:
            item01 = GP01Item()
            for key, value in data_dict.items():
                try:
                    item01[key] = value
                except:
                    continue
            item01['isFind'] = True

            # 获取楼盘详情
            yield scrapy.Request(
                self.gp02_url.format(item01['id']),
                callback=self.parse_detail,
                meta=dict(loupan_name=deepcopy(loupan_name), district_name=deepcopy(district_name), ),
                dont_filter=True,
            )

            # 获取楼栋列表
            yield scrapy.Request(
                self.gp03_url.format(item01['id']),
                callback=self.parse_building_li,
                meta=dict(loupan_name=deepcopy(loupan_name), district_name=deepcopy(district_name), ),
                dont_filter=True,
            )

            # 获取住宅类别
            yield scrapy.Request(
                self.gp04_url.format(item01['id']),
                callback=self.parse_residence_type,
                meta=dict(loupan_name=deepcopy(loupan_name), district_name=deepcopy(district_name),
                          id=deepcopy(item01['id']), ),
                dont_filter=True,
            )

            # 获取面积段均价
            yield scrapy.Request(
                self.gp05_url.format(item01['id']),
                callback=self.parse_avg_price,
                meta=dict(loupan_name=deepcopy(loupan_name), district_name=deepcopy(district_name),
                          id=deepcopy(item01['id']), ),
                dont_filter=True,
            )

            # 获取待建楼盘
            yield scrapy.Request(
                self.gp06_url.format(quote(loupan_name)),
                callback=self.parse_built,
                meta=dict(loupan_name=deepcopy(loupan_name), district_name=deepcopy(district_name),
                          item01=deepcopy(item01), ),
                dont_filter=True,
            )

    def parse_detail(self, response):
        """
        获取楼盘详情
        :param respone:
        :return:
        """
        district_name = copy(response.meta['district_name'])
        loupan_name = copy(response.meta['loupan_name'])
        try:
            resp = json.loads(response.body.decode())
            data_dict = resp['data']
            assert data_dict
        except:
            logger.error(f'{district_name}-{loupan_name} 楼盘详情获取出错')
        else:
            item02 = GP02Item()
            for key, value in data_dict.items():
                try:
                    item02[key.strip()] = value
                except:
                    continue
            yield item02

    def parse_building_li(self, response):
        """
        获取楼栋列表
        :param respone:
        :return:
        """
        district_name = copy(response.meta['district_name'])
        loupan_name = copy(response.meta['loupan_name'])
        try:
            resp = json.loads(response.body.decode())
            data_list = resp['data']
            assert data_list
        except:
            logger.error(f'{district_name}-{loupan_name} 楼栋列表获取出错')
        else:
            for data_dict in data_list:
                item03 = GP03Item()
                for key, value in data_dict.items():
                    try:
                        item03[key] = value
                    except:
                        continue
                yield item03

    def parse_residence_type(self, response):
        """
        获取住宅类别
        :param respone:
        :return:
        """
        district_name = copy(response.meta['district_name'])
        loupan_name = copy(response.meta['loupan_name'])
        id = copy(response.meta['id'])
        try:
            resp = json.loads(response.body.decode())
            data_list = resp['data']
            assert data_list
        except:
            logger.error(f'{district_name}-{loupan_name} 住宅类别获取出错')
        else:
            for data_dict in data_list:
                item04 = GP04Item()
                item04['communityName'] = loupan_name
                item04['communityId'] = id
                for key, value in data_dict.items():
                    try:
                        item04[key] = value
                    except:
                        continue
                yield item04

    def parse_avg_price(self, response):
        """
        获取面积段均价
        :param response:
        :return:
        """
        district_name = copy(response.meta['district_name'])
        loupan_name = copy(response.meta['loupan_name'])
        try:
            resp = json.loads(response.body.decode())
            data_list = resp['data']
            assert data_list
        except:
            logger.error(f'{district_name}-{loupan_name} 面积段均价获取出错')
        else:
            for data_dict in data_list:
                item05 = GP05Item()
                for key, value in data_dict.items():
                    try:
                        item05[key] = value
                    except:
                        continue
                yield item05

    def parse_built(self, response):
        """
        获取待建楼盘
        :param response:
        :return:
        """
        item01 = copy(response.meta['item01'])
        try:
            resp = json.loads(response.body.decode())
            data_list = resp['data']
            assert data_list
        except:
            item01['isBuilt'] = False
        else:
            item01['isBuilt'] = True
        finally:
            yield item01
